const axios = require('axios')
const iconv = require('iconv-lite')
const cheerio = require('cheerio')
const fs = require('fs')

axios({
    method: 'get',
    url: 'https://www.81book.com/',
    responseType: 'stream'
}).then(
    res => {
        let stream = [];
        res.data.on('data', st => {
            stream.push(st);
        });
        res.data.on('end', () => {
            let buffer = Buffer.concat(stream);
            let domStr = iconv.decode(buffer, 'gbk');
            var $ = cheerio.load(domStr)
            var arr = []
            $('a').each(function() {
                var href = $(this).attr('href');
                // 筛选和去重
                if (href.slice(0, 6) === '/book/' && href.slice(-4) != 'html' && arr.indexOf(href) < 0) {
                    arr.push(href)
                }
            });

            var arrStr = JSON.stringify(arr)
            fs.writeFile('./url.json', arrStr, err => {
                if (err) {
                    console.log(err)
                } else {
                    console.log('数据已经存储到url.json')
                }
            })
        })
    }
).catch(err=>{
    console.log(err)
})